package com.stack.wemedia.test;

import com.alibaba.fastjson.JSON;
import com.stack.apis.admin.IAdminClient;
import com.stack.common.utils.thread.WmThreadLocalUtil;
import com.stack.file.service.FileStorageService;
import com.stack.model.admin.dtos.AdNewsDto;
import com.stack.model.wemedia.dtos.WmNewsDto;
import com.stack.model.wemedia.pojos.NewsCrawler;
import com.stack.model.wemedia.pojos.WmMaterial;
import com.stack.model.wemedia.pojos.WmUser;
import com.stack.wemedia.WemediaApplication;
import com.stack.wemedia.mapper.WmMaterialMapper;
import com.stack.wemedia.service.NewsCrawlerService;
import com.stack.wemedia.service.WmNewsService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.tomcat.util.http.fileupload.IOUtils;
import org.assertj.core.util.Lists;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

import javax.annotation.Resource;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.*;

/**
 * @author llz
 * @date 2023/11/30 9:31
 * @description ZhiBo8爬取新闻数据
 */
@SpringBootTest(classes = WemediaApplication.class)
@RunWith(SpringRunner.class)
@Slf4j
public class ZhiBo8Crawler {
    @Resource
    private NewsCrawlerService newsCrawlerService;
    @Resource
    private WmNewsService wmNewsService;
    @Resource
    private WmMaterialMapper wmMaterialMapper;
    @Resource
    private FileStorageService fileStorageService;
    @Resource
    private IAdminClient adminClient;

    @Test
    public void test() {
//        List<String> materials = new ArrayList<>();
//        materials.add("http://192.168.200.130:9000/leadnews/2023/12/01/316558ee5cbf4b8db84c1f0e601b8682.gif");
//        List<WmMaterial> dbMaterials = wmMaterialMapper.selectList(Wrappers.<WmMaterial>lambdaQuery().in(WmMaterial::getUrl, materials));
//        log.info("通过图片的url查询素材的id：{}", dbMaterials);

        List<String> urls = newsCrawlerService.getUrlByChannel(1);
        log.info("获取的url：{}", urls);
        for (String url : urls) {
            NewsCrawler newsCrawler = new NewsCrawler();
            newsCrawler.setUrl(url);
            newsCrawler.setStatus(1);
            newsCrawlerService.update(newsCrawler);
        }

//        NewsCrawler newsCrawler = new NewsCrawler();
//        newsCrawler.setTitle("title");
//        newsCrawler.setUrl("newsUrl");
//        newsCrawler.setChannel(1);
//        newsCrawler.setStatus(0);
//        newsCrawler.setCreatedTime(new Date());
//        log.info("插入数据:{}", JSON.toJSONString(newsCrawler));
//        Integer i = newsCrawlerService.insert(newsCrawler);
//        log.info("结果:{}", i);
    }

    /**
     * 获取频道每条新闻的url
     */
    @Test
    public void getNewsUrl() throws IOException {
        // 获取网易新闻数据
//        String url = "https://3g.163.com/touch/news/sub/history/?ver=c&clickfrom=index2018_header_main";
//        String url = "https://3g.163.com/touch/news/?ver=c&clickfrom=index2018_header_main";
//        String url = "https://3g.163.com/touch/ent/sub/television/?ver=c&clickfrom=index2018_header_main";
        // 直播吧
        String url = "https://m.zhibo8.com/news.htm";

        // 获取该网页document文档数据
        Document document = Jsoup.connect(url)
                .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36")
                .get();

        log.info("document: {}", document);

        // 找到指定class的所有元素
//        Elements elementList = document.getElementsByClass("icon_video");
//        log.info("div元素: {}", elementList.size());

        Elements elements = document.getElementsByTag("h2");
        log.info("h2元素: {}", elements.size());
        for (Element element : elements) {
            Elements tag = element.getElementsByTag("a");
            if (tag.size() == 0) {
                continue;
            }
            String newsUrl = "https://m.zhibo8.com" + tag.attr("href");
            String title = element.text();
//            if (title.length() > 30) {
//                title = title.substring(0, 29);
//            }

            log.info("uri: {}", tag.attr("href"));
            log.info("新闻的url: {}", newsUrl);
            log.info("标题: {}", title);

            try {
                NewsCrawler newsCrawler = new NewsCrawler();
                newsCrawler.setTitle(title);
                newsCrawler.setUrl(newsUrl);
                newsCrawler.setChannel(1);
                newsCrawler.setStatus(0);
                newsCrawler.setCreatedTime(new Date());
                newsCrawlerService.insert(newsCrawler);
                log.info("爬爬爬爬爬爬爬!");
            } catch (Exception e) {
                log.error("标题不合规，爬取失败: {}", e.getMessage());
            }
        }
    }

    /**
     * 获取新闻细节
     */
    @Test
    public void getNewsDetail() throws IOException {
        // 模拟当前自媒体登录用户
        WmUser wmUser = new WmUser();
        wmUser.setId(1103);
        WmThreadLocalUtil.setUser(wmUser);

        List<String> urls = newsCrawlerService.getUrlByChannel(1);
        log.info("本次爬取的url数量: {}", urls.size());
        for (String url : urls) {
            log.info("本次爬取url: {}", url);
            if (url.length() < "https://m.zhibo8.comnews".length()) {
                continue;
            }

            // 获取该网页document文档数据
            Document document = null;
            try {
                document = Jsoup.connect(url)
                        .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36")
                        .get();
                log.info("document: {}", document);
            } catch (Exception e) {
                log.error("获取新闻详情失败,原因是{}", e.getMessage());
                continue;
            }


            String title = document.getElementsByClass("title").text();
            log.info("新闻的标题: {}", title);
//            if (title.length() > 30) {
//                title = title.substring(0, 29);
//            }

            if (document.getElementsByClass("img_wrap").size() <= 0) {
                continue;
            }

            List<String> imgUrlList = new ArrayList<>();
            Elements imgElements = document.getElementsByClass("img_wrap");
            for (Element imgElement : imgElements) {
                String imgUrl = imgElement.attr("data");
                String fileName = uploadPic(imgUrl);
                imgUrlList.add(fileName);
            }

//            List<String> coverImgList = new ArrayList<>();
//            String coverImg = document.getElementsByClass("img_wrap").get(0).attr("data");
//            log.info("新闻的封面图片: {}", coverImg);
//            String coverUrl = uploadPic(coverImg);
//            coverImgList.add(coverUrl);


            // 上传图片//保存素材
            for (String imgUrl : imgUrlList) {
                WmMaterial wmMaterial = null;
                if (StringUtils.isNotBlank(imgUrl)) {
                    // 如果上传图片路径不为空  创建素材信息
                    wmMaterial = new WmMaterial();
                    wmMaterial.setUserId(WmThreadLocalUtil.getUser().getId());
                    wmMaterial.setUrl(imgUrl);
                    wmMaterial.setType((short) 0);
                    wmMaterial.setIsCollection((short) 0);
                    wmMaterial.setCreatedTime(new Date());
                    log.info("插入素材信息: {}", wmMaterial);
                    wmMaterialMapper.insert(wmMaterial);
                }
            }

            try {
                // 保存自媒体文章
                List<Map> contentMap = parseContents(document, imgUrlList);
                log.info("解析后的内容: {}", contentMap);
                WmNewsDto wmNewsDto = new WmNewsDto();
                wmNewsDto.setContent(JSON.toJSONString(contentMap));
                wmNewsDto.setTitle(title);
                wmNewsDto.setType((short) 1);
                if (imgUrlList.size() > 0) {
                    List<String> imgUrls = Lists.newArrayList(imgUrlList.get(0));
                    wmNewsDto.setImages(imgUrls);
                }
                wmNewsDto.setChannelId(5);
                try {
                    Thread.sleep(1000); // 睡眠1秒 让发布时间不一致
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
                wmNewsDto.setPublishTime(new Date());
                wmNewsDto.setStatus((short) 9); // 已发布状态
                wmNewsDto.setLabels("采集");
                log.info("保存自媒体文章: {}", wmNewsDto);
                wmNewsService.submitNews(wmNewsDto);

                // 保存admin端的相关的审核文章数据
                AdNewsDto dto = new AdNewsDto();
                dto.setUserId(WmThreadLocalUtil.getUser().getId());
                dto.setAuthorName("crawler菜鸟");
                dto.setTitle(title);
                dto.setContent(JSON.toJSONString(contentMap));
                dto.setStatus(3);
                dto.setChannelId(0);
                dto.setLabels("采集");
                dto.setCreatedTime(new Date());
                dto.setSubmitedTime(new Date());
                dto.setType(1);
                if (imgUrlList.size() > 0) {
                    List<String> imgUrls = Lists.newArrayList(imgUrlList.get(0));
                    dto.setImages(imgUrls);
                }
                dto.setEnable(0);
                log.info("保存admin端的相关的审核文章数据: {}", dto);
                adminClient.saveAdCheckNews(dto);
            } catch (Exception e) {
                log.error("保存自媒体文章失败");
                log.error("保存admin端的相关的审核文章数据失败");
                continue;
            } finally {
                //修改爬虫配置表
                NewsCrawler newsCrawler = new NewsCrawler();
                newsCrawler.setUrl(url);
                newsCrawler.setStatus(1);
                newsCrawlerService.update(newsCrawler);
            }
        }

    }


    private List<Map> parseContents(Document document, List<String> imgUrlList) {
        List<Map> contentMap = new ArrayList<>();
        try {
            Elements elements = document.getElementsByTag("p");
            Map<String, String> map = null;
            String text = null;
            for (Element element : elements) {
                if (StringUtils.isNotBlank(element.text())) {
                    text = text + "\n" + element.text();
                }
            }
            map = new HashMap<>();
            map.put("type", "text");
            map.put("value", text);
            contentMap.add(map);

            for (String imgUrl : imgUrlList) {
                map = new HashMap<>();
                map.put("type", "image");
                map.put("value", imgUrl);
                contentMap.add(map);
            }
        } catch (Exception e) {
            e.printStackTrace();
            Map<String, String> map = new HashMap<>();
            map.put("type", "text");
            map.put("value", "测试文章内容");
            contentMap.add(map);
        }
        return contentMap;
    }

    public String uploadPic(String imgUrl) {
        String uuid = UUID.randomUUID().toString().replace("-", "");
        String suffix = imgUrl.substring(imgUrl.lastIndexOf("."));
        InputStream in = getInputStreamByUrl(imgUrl);
        if (in != null) {
            String fileName = fileStorageService.uploadImgFile("", uuid + suffix, in);
            System.out.println("上传文件名称: " + fileName);
            return fileName;
        }
        return null;
    }

    /**
     * 工具方法:  根据url路径 获取输入流
     *
     * @param strUrl
     * @return
     */
    public static InputStream getInputStreamByUrl(String strUrl) {
        HttpURLConnection conn = null;
        try {
            URL url = new URL(strUrl);
            conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("GET");
            conn.setConnectTimeout(20 * 1000);
            final ByteArrayOutputStream output = new ByteArrayOutputStream();
            IOUtils.copy(conn.getInputStream(), output);
            return new ByteArrayInputStream(output.toByteArray());
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                if (conn != null) {
                    conn.disconnect();
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        return null;
    }

    //
//    private void getNewsDetails(List<Element> articleList) {
//        // 遍历articleList
//        if (CollectionUtils.isNotEmpty(articleList)) {
//            for (Element article : articleList) {
//                Element element = article.getElementsByTag("a").get(0);
//                String href = element.attr("href");
//                log.info("新闻的url路径: {}", href);
//                Element titleEle = article.getElementsByTag("h2").get(0);
//                String text = titleEle.getElementsByTag("a").text();
//                log.info("新闻的标题: {}", text);
//                String title = titleEle.text();
//                log.info("新闻的标题: {}", title);
//                String picture = article.attr("news-pic");
//                log.info("新闻的封面图片: {}", picture);
//                Element newsPic = article.getElementsByClass("news-pic").get(0);
//                // 获取封面图片元素集合
//                Elements imgList = newsPic.getElementsByTag("img");
//                // 封装WmNewsDto对象
//                WmNewsDto wmNewsDto = new WmNewsDto();
//                // 解析单个文章详情
//                List<Map> contentMap = parseContent(href);
//                wmNewsDto.setContent(JSON.toJSONString(contentMap));
//                // 封面图片集合
//                List<String> urlList = new ArrayList<>();
//                for (Element imgEle : imgList) {
//                    String src = imgEle.attr("src");
//                    System.out.println("封面图片url==>" + src);
//                    String fileName = uploadPic(src);
//                    if (StringUtils.isNotBlank(fileName)) {
//                        // 如果上传图片路径不为空  创建素材信息
//                        WmMaterial wmMaterial = new WmMaterial();
//                        wmMaterial.setUserId(WmThreadLocalUtil.getUser().getId());
//                        wmMaterial.setUrl(fileName);
//                        wmMaterial.setType((short) 0);
//                        wmMaterial.setIsCollection((short) 0);
//                        wmMaterial.setCreatedTime(new Date());
//                        log.info("插入素材信息: {}", wmMaterial);
//                        wmMaterialMapper.insert(wmMaterial);
//                        urlList.add(fileName);
//                    }
//                }
//                wmNewsDto.setTitle(title);
//                wmNewsDto.setType((short) urlList.size());
//                if (urlList.size() > 0) {
//                    wmNewsDto.setImages(urlList);
//                }
//                wmNewsDto.setChannelId(6);
//                try {
//                    Thread.sleep(1000); // 睡眠1秒 让发布时间不一致
//                } catch (InterruptedException e) {
//                    e.printStackTrace();
//                }
//                wmNewsDto.setPublishTime(new Date());
//                wmNewsDto.setStatus((short) 1); // 待审核状态
//                wmNewsDto.setLabels("采集");
//                wmNewsService.submitNews(wmNewsDto);
//            }
//        }
//    }

//    /**
//     * 解析文章详情内容
//     *
//     * @param href
//     * @return
//     */
//    private List<Map> parseContent(String href) {
//        String url = "http:" + href;
//        List<Map> contentMap = new ArrayList<>();
//        Document document = null;
//        try {
//            document = Jsoup.connect(url)
//                    .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36")
//                    .get();
//            Element contentDiv = document.getElementsByClass("content").get(0);
//            Element pageDiv = contentDiv.getElementsByTag("div").get(0);
//            Elements allElements = pageDiv.getAllElements();
//
//            for (Element subElement : allElements) {
//                String tagName = subElement.tagName();
//                String className = subElement.className();
//                if ("p".equalsIgnoreCase(tagName)) {
//                    Map map = new HashMap();
//                    map.put("type", "text");
//                    map.put("value", subElement.text());
//                    contentMap.add(map);
//                    System.out.println("文本内容: " + subElement.text());
//                }
//                if ("div".equalsIgnoreCase(tagName) && "photo".equalsIgnoreCase(className)) {
//                    System.out.println("图片内容: " + subElement.text());
//                }
//            }
//        } catch (Exception e) {
//            e.printStackTrace();
//            Map map = new HashMap();
//            map.put("type", "text");
//            map.put("value", "测试文章内容");
//            contentMap.add(map);
//        }
//        return contentMap;
//    }

}